{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a5079bf3-97cd-40f3-ba6a-35cd662f7439",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from datasets import load_dataset\n",
    "from swebench import MAP_VERSION_TO_INSTALL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "93b467a6-a450-49e7-9283-5bcb520f7f23",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_dataset(\"princeton-nlp/SWE-bench\", split=\"test\")\n",
    "\n",
    "# NOTE: We have not released the gold predictions, so this is just a placeholder and will not work\n",
    "golds = [json.loads(x) for x in open(\"gold_preds.jsonl\")]\n",
    "golds = {x['instance_id']: x for x in golds}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "889859d0-f5e7-4670-a133-0dc8fb1cdf75",
   "metadata": {},
   "outputs": [],
   "source": [
    "repo_version_pairs = [\n",
    "    (repo, version)\n",
    "    for repo, version_map in MAP_VERSION_TO_INSTALL.items()\n",
    "    for version in version_map.keys()\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b9aaa3d4-4d8f-4b8c-b516-61c98ab0ccde",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "126"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_harness = []\n",
    "for repo, version in repo_version_pairs:\n",
    "    subset = [x for x in data if x['repo'] == repo and x['version'] == version]\n",
    "    if len(subset) == 0:\n",
    "        continue\n",
    "    subset = sorted(subset, key=lambda x: x['created_at'], reverse=False)\n",
    "    inst_id = subset[-1]['instance_id']\n",
    "    check_harness.append(golds[inst_id])\n",
    "len(check_harness)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "70591c07-8cc3-4f7f-800a-07eec5d4e5ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"check-harness.jsonl\", \"w\") as f:\n",
    "    for gold_pred in check_harness:\n",
    "        print(json.dumps(gold_pred), file=f, flush=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "651f2c61-ed58-403c-ac6a-71f5375e8b59",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
